{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NinaPro动作分类——多模态学习方法\n",
    "\n",
    "LMF（Low-rank Multimodal Fusion）出自论文 Efficient Low-rank Multimodal Fusion with Modality-Specific Factors，ACL2018。是TFN的等价升级版，就具体模型如图。LMF利用对权重进行低秩矩阵分解，将TFN先张量外积再FC的过程变为每个模态先单独线性变换之后再多维度点积，可以看作是多个低秩向量的结果的和，从而减少了模型中的参数数量。"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.数据读取与预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import io\n",
    "import numpy as np\n",
    "\n",
    "def get_feature_dict(filename):\n",
    "    \"\"\"将ninapro_feature的MAT文件加载为字典\n",
    "\n",
    "    Args:\n",
    "        path: mat文件路径\n",
    "\n",
    "    Returns:\n",
    "        数据集字典\n",
    "        [feat_set, featStim, featRep]\n",
    "    \"\"\"\n",
    "    # 读取MAT文件\n",
    "    print('load file: ' + filename + '...', end= '', flush=True)\n",
    "    dict_feature=io.loadmat(filename)\n",
    "    if (dict_feature != ()):\n",
    "        #print(ninapro_data.keys())\n",
    "        print('[ok]:%d'%(len(dict_feature['featStim'])), flush=True)\n",
    "    # 返回字典\n",
    "    return dict_feature\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load file: ../feature/feature_S1.mat...[ok]:206751\n",
      "load file: ../feature/IMU_feature_S1.mat...[ok]:206751\n",
      "load file: ../feature/feature_S2.mat...[ok]:206659\n",
      "load file: ../feature/IMU_feature_S2.mat...[ok]:206659\n",
      "(97210, 120, 5) (97210, 1)\n"
     ]
    }
   ],
   "source": [
    "def split_zeros(feature_dict,feature_name,channels):\n",
    "    \"\"\"将ninapro_feature数据集中【restimulate】为0的部分（受试者不做动作）从数据集中去除\n",
    "\n",
    "    Args:\n",
    "        feature_dict: 数据集字典\n",
    "        feature_name: 待处理的数据的keyvalue\n",
    "        channels: 待处理的数据的通道数\n",
    "\n",
    "    Returns:\n",
    "        [feature_split, labels] 去除0部分的数据，对于的label(numpy array)\n",
    "    \"\"\"\n",
    "    feature_split = None\n",
    "    index = []\n",
    "    for i in range(len(feature_dict['featStim'])):\n",
    "        if feature_dict['featStim'][i]!=0:\n",
    "            index.append(i)\n",
    "    # 重排元素\n",
    "    emg_temp = feature_dict[feature_name]\n",
    "    emg_temp = np.reshape(emg_temp,(-1,5,channels))\n",
    "    emg_temp = np.swapaxes(emg_temp,1,2)\n",
    "    # 去除0label\n",
    "    if(feature_split is None):\n",
    "        feature_split = emg_temp[index,:,:]\n",
    "        labels = feature_dict['featStim'][index,:]\n",
    "    else:\n",
    "        feature_split = np.vstack((feature_split,emg_temp[index,:,:])) \n",
    "        labels = np.vstack((labels,feature_dict['featStim'][index,:]))\n",
    "    return feature_split, labels\n",
    "\n",
    "# 对多组数据合并，预处理\n",
    "def merge_multisubject(b,e):\n",
    "    \"\"\"将多组数据从mat文件中提取出来，预处理后合并\n",
    "\n",
    "    Args:\n",
    "        b: 开始的受试者序号\n",
    "        e: 结束的受试者序号\n",
    "\n",
    "    Returns:\n",
    "        [emg,acc,gyro,mag,labels]肌电c12，加速度c36，角速度c36，磁强c36数据和标签。\n",
    "    \"\"\"\n",
    "    emg_feature = None\n",
    "    labels = None\n",
    "    # 遍历受试者序号\n",
    "    for i in range(b,e+1):\n",
    "        emg_dict = get_feature_dict(\"../feature/feature_S{0}.mat\".format(i))\n",
    "        imu_dict = get_feature_dict(\"../feature/IMU_feature_S{0}.mat\".format(i))\n",
    "        # 寻找动作为0的元素并剔除\n",
    "        emg,labels = split_zeros(emg_dict,'feat_set',12)\n",
    "        acc,labels = split_zeros(imu_dict,'acc_feat',36)\n",
    "        gyro,labels = split_zeros(imu_dict,'gyro_feat',36)\n",
    "        mag,labels = split_zeros(imu_dict,'mag_feat',36)\n",
    "        #print('delete 0 label,',emg_temp[index,:,:].shape)\n",
    "    # s = [1.28889041e-05, 0.00000000e+00, 1.72402617e+01, 1.57331247e+01, 2.11883893e-03]\n",
    "    # 归一化\n",
    "    # for i in range(5):\n",
    "    #     #s[i] = np.sum(np.abs(emg_feature[:,:,i]))/emg_feature[:,:,i].size\n",
    "    #     #print(\"avg=\",s)\n",
    "    #     if(s[i]!=0):\n",
    "    #         emg_feature[:,:,i] /= s[i]\n",
    "    #         emg_feature[:,:,i] -= 0.5*s[i]\n",
    "    return emg,acc,gyro,mag,labels\n",
    "\n",
    "# 读取2组数据分布作训练集和验证集\n",
    "emg_feature,acc_feature,gyro_feature,mag_feature,labels = merge_multisubject(1,1)\n",
    "feat_set = np.concatenate((emg_feature,acc_feature,gyro_feature,mag_feature),axis = 1)\n",
    "\n",
    "emg_feature,acc_feature,gyro_feature,mag_feature,labels_test = merge_multisubject(2,2)\n",
    "feat_set_test = np.concatenate((emg_feature,acc_feature,gyro_feature,mag_feature),axis = 1)\n",
    "print(feat_set.shape,labels.shape)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(97210, 1, 5, 120) (97210,)\n",
      "(105147, 1, 5, 120) (105147,)\n"
     ]
    }
   ],
   "source": [
    "#数据预处理\n",
    "def precess_data(feat,label):\n",
    "    \"\"\"将数据reshape到可以送入神经网络的size\n",
    "\n",
    "    Args:\n",
    "        feat: 特征序列\n",
    "        label: 标签\n",
    "\n",
    "    Returns:\n",
    "        [feat, label] 处理后的特征序列和标签\n",
    "    \"\"\"\n",
    "    feat = np.swapaxes(feat,1,2)\n",
    "    feat = np.expand_dims(feat,1)\n",
    "    feat = feat.astype(np.float32)\n",
    "    label = label.flatten() - 1\n",
    "    return feat,label\n",
    "feat_set,labels = precess_data(feat_set,labels)\n",
    "feat_set_test,labels_test = precess_data(feat_set_test,labels_test)\n",
    "\n",
    "print(feat_set.shape,labels.shape)\n",
    "print(feat_set_test.shape,labels_test.shape)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.模型建立"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model will be running on cuda:0 device\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1            [-1, 64, 3, 60]           3,136\n",
      "       BatchNorm2d-2            [-1, 64, 3, 60]             128\n",
      "              ReLU-3            [-1, 64, 3, 60]               0\n",
      "         MaxPool2d-4            [-1, 64, 2, 30]               0\n",
      "            Conv2d-5            [-1, 64, 2, 30]          36,864\n",
      "       BatchNorm2d-6            [-1, 64, 2, 30]             128\n",
      "              ReLU-7            [-1, 64, 2, 30]               0\n",
      "            Conv2d-8            [-1, 64, 2, 30]          36,864\n",
      "       BatchNorm2d-9            [-1, 64, 2, 30]             128\n",
      "             ReLU-10            [-1, 64, 2, 30]               0\n",
      "       BasicBlock-11            [-1, 64, 2, 30]               0\n",
      "           Conv2d-12            [-1, 64, 2, 30]          36,864\n",
      "      BatchNorm2d-13            [-1, 64, 2, 30]             128\n",
      "             ReLU-14            [-1, 64, 2, 30]               0\n",
      "           Conv2d-15            [-1, 64, 2, 30]          36,864\n",
      "      BatchNorm2d-16            [-1, 64, 2, 30]             128\n",
      "             ReLU-17            [-1, 64, 2, 30]               0\n",
      "       BasicBlock-18            [-1, 64, 2, 30]               0\n",
      "           Conv2d-19           [-1, 128, 1, 15]          73,728\n",
      "      BatchNorm2d-20           [-1, 128, 1, 15]             256\n",
      "             ReLU-21           [-1, 128, 1, 15]               0\n",
      "           Conv2d-22           [-1, 128, 1, 15]         147,456\n",
      "      BatchNorm2d-23           [-1, 128, 1, 15]             256\n",
      "           Conv2d-24           [-1, 128, 1, 15]           8,192\n",
      "      BatchNorm2d-25           [-1, 128, 1, 15]             256\n",
      "             ReLU-26           [-1, 128, 1, 15]               0\n",
      "       BasicBlock-27           [-1, 128, 1, 15]               0\n",
      "           Conv2d-28           [-1, 128, 1, 15]         147,456\n",
      "      BatchNorm2d-29           [-1, 128, 1, 15]             256\n",
      "             ReLU-30           [-1, 128, 1, 15]               0\n",
      "           Conv2d-31           [-1, 128, 1, 15]         147,456\n",
      "      BatchNorm2d-32           [-1, 128, 1, 15]             256\n",
      "             ReLU-33           [-1, 128, 1, 15]               0\n",
      "       BasicBlock-34           [-1, 128, 1, 15]               0\n",
      "           Conv2d-35            [-1, 256, 1, 8]         294,912\n",
      "      BatchNorm2d-36            [-1, 256, 1, 8]             512\n",
      "             ReLU-37            [-1, 256, 1, 8]               0\n",
      "           Conv2d-38            [-1, 256, 1, 8]         589,824\n",
      "      BatchNorm2d-39            [-1, 256, 1, 8]             512\n",
      "           Conv2d-40            [-1, 256, 1, 8]          32,768\n",
      "      BatchNorm2d-41            [-1, 256, 1, 8]             512\n",
      "             ReLU-42            [-1, 256, 1, 8]               0\n",
      "       BasicBlock-43            [-1, 256, 1, 8]               0\n",
      "           Conv2d-44            [-1, 256, 1, 8]         589,824\n",
      "      BatchNorm2d-45            [-1, 256, 1, 8]             512\n",
      "             ReLU-46            [-1, 256, 1, 8]               0\n",
      "           Conv2d-47            [-1, 256, 1, 8]         589,824\n",
      "      BatchNorm2d-48            [-1, 256, 1, 8]             512\n",
      "             ReLU-49            [-1, 256, 1, 8]               0\n",
      "       BasicBlock-50            [-1, 256, 1, 8]               0\n",
      "           Conv2d-51            [-1, 512, 1, 4]       1,179,648\n",
      "      BatchNorm2d-52            [-1, 512, 1, 4]           1,024\n",
      "             ReLU-53            [-1, 512, 1, 4]               0\n",
      "           Conv2d-54            [-1, 512, 1, 4]       2,359,296\n",
      "      BatchNorm2d-55            [-1, 512, 1, 4]           1,024\n",
      "           Conv2d-56            [-1, 512, 1, 4]         131,072\n",
      "      BatchNorm2d-57            [-1, 512, 1, 4]           1,024\n",
      "             ReLU-58            [-1, 512, 1, 4]               0\n",
      "       BasicBlock-59            [-1, 512, 1, 4]               0\n",
      "           Conv2d-60            [-1, 512, 1, 4]       2,359,296\n",
      "      BatchNorm2d-61            [-1, 512, 1, 4]           1,024\n",
      "             ReLU-62            [-1, 512, 1, 4]               0\n",
      "           Conv2d-63            [-1, 512, 1, 4]       2,359,296\n",
      "      BatchNorm2d-64            [-1, 512, 1, 4]           1,024\n",
      "             ReLU-65            [-1, 512, 1, 4]               0\n",
      "       BasicBlock-66            [-1, 512, 1, 4]               0\n",
      "AdaptiveAvgPool2d-67            [-1, 512, 1, 1]               0\n",
      "           Linear-68                   [-1, 40]          20,520\n",
      "           ResNet-69                   [-1, 40]               0\n",
      "================================================================\n",
      "Total params: 11,190,760\n",
      "Trainable params: 11,190,760\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 1.44\n",
      "Params size (MB): 42.69\n",
      "Estimated Total Size (MB): 44.13\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torchsummary import summary\n",
    "import torch.nn.functional as F\n",
    "from torchvision import models\n",
    "from torch.nn.parameter import Parameter\n",
    "\n",
    "class LFM_net(nn.Module):\n",
    "    \"\"\"LFM实现的卷积神经网络\n",
    "    \"\"\"\n",
    "    def __init__(self, class_num=40):\n",
    "        super(LFM_net, self).__init__()\n",
    "        self.class_num = class_num\n",
    "\n",
    "        self.res18 = models.resnet18(num_classes=class_num)\n",
    "        self.res18.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n",
    "\n",
    "    def LFM_fushion(self, x, R = 4, h = 24):\n",
    "        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "        A = x[:,:,0:12].view(5,12).to(device)\n",
    "        B = x[:,:,12:].view(5,108).to(device)\n",
    "        \n",
    "        n = A.shape[0]\n",
    "        A = torch.cat([A, torch.ones(n, 1).to(device)], dim=1).float().to(device)\n",
    "        B = torch.cat([B, torch.ones(n, 1).to(device)], dim=1).float().to(device)\n",
    "\n",
    "        \n",
    "        \n",
    "        Wa = Parameter(torch.Tensor(R, A.shape[1], h)).to(device)\n",
    "        Wb = Parameter(torch.Tensor(R, B.shape[1], h)).to(device)\n",
    "        Wf = Parameter(torch.Tensor(1, R)).to(device)\n",
    "        bias = Parameter(torch.Tensor(1, h)).to(device)\n",
    "\n",
    "        fusion_A = torch.matmul(A, Wa).to(device)\n",
    "        fusion_B = torch.matmul(B, Wb).to(device)\n",
    "\n",
    "        funsion_AB = fusion_A * fusion_B\n",
    "        funsion_AB = torch.matmul(Wf, funsion_AB.permute(1,0,2)).squeeze() + bias\n",
    "        funsion_AB = funsion_AB.unsqueeze(0)\n",
    "        return funsion_AB    \n",
    "\n",
    "    def forward(self, x):\n",
    "        #print(x.size())\n",
    "        xf = torch.zeros(x.shape[0],5,24)\n",
    "        for i in range(x.shape[0]):\n",
    "            xf[i] = self.LFM_fushion(x[i],R=4,h=24)\n",
    "        # 假设所设秩: R = 4, 期望融合后的特征维度: h = 24\n",
    "        x = self.res18(x)\n",
    "        return x\n",
    "\n",
    "# 输出网络结构\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(\"The model will be running on\", device, \"device\")\n",
    "model_test = LFM_net(class_num=40).to(device)\n",
    "summary(model_test,(1,5,120))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "def testAccuracy(device, model, test_loader):\n",
    "    \n",
    "    model.eval()\n",
    "    accuracy = 0.0\n",
    "    total = 0.0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for data in test_loader:\n",
    "            emg_data, labels = data\n",
    "            emg_data = Variable(emg_data.to(device))    # torch.Size([64, 1, 200, 12])\n",
    "            labels = Variable(labels.to(device))        # torch.Size([64])\n",
    "            # run the model on the test set to predict labels\n",
    "            outputs = model(emg_data)\n",
    "            # the label with the highest energy will be our prediction\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            accuracy += (predicted == labels).sum().item()\n",
    "    \n",
    "    # compute the accuracy over all test images\n",
    "    accuracy = (100 * accuracy / total)\n",
    "    return(accuracy)\n",
    "\n",
    "train_accs = []\n",
    "train_loss = []\n",
    "test_accs = []\n",
    "\n",
    "def train(device, num_epochs, train_loader, test_loader):\n",
    "    best_accuracy = 0\n",
    "    # model = models.resnet18(num_classes=40)\n",
    "    # model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n",
    "    # model = model.to(device)\n",
    "    model = LFM_net(class_num=40).to(device)\n",
    "    loss_func = torch.nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "\n",
    "    \n",
    "    \n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for i,(inputs, labels) in enumerate(train_loader,0):#0是下标起始位置默认为0\n",
    "            \n",
    "            inputs = Variable(inputs.to(device))    \n",
    "            labels = Variable(labels.to(device))    \n",
    "            #初始为0，清除上个batch的梯度信息\n",
    "            optimizer.zero_grad()         \n",
    "\n",
    "            #前向+后向+优化     \n",
    "            outputs = model(inputs)\n",
    "            loss = loss_func(outputs,labels.long())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # loss 的输出，每个一百个batch输出，平均的loss\n",
    "            running_loss += loss.item()\n",
    "            if i%100 == 99:\n",
    "                print('[%d,%5d] loss :%.3f' %\n",
    "                    (epoch+1,i+1,running_loss/100),end='',flush=True)\n",
    "                running_loss = 0.0\n",
    "            train_loss.append(loss.item())\n",
    "\n",
    "            # 训练曲线的绘制 一个batch中的准确率\n",
    "            correct = 0\n",
    "            total = 0\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total = labels.size(0)# labels 的长度\n",
    "            correct = (predicted == labels).sum().item() # 预测正确的数目\n",
    "            train_accs.append(100*correct/total)\n",
    "            if i%100 == 99:\n",
    "                print(' acc=%d'%(100*correct/total))\n",
    "            \n",
    "        accuracy = testAccuracy(device, model, test_loader)\n",
    "        print('For epoch', epoch+1,'the test accuracy over the whole test set is %d %%' % (accuracy))\n",
    "        # we want to save the model if the accuracy is the best\n",
    "        if accuracy > best_accuracy:\n",
    "            best_accuracy = accuracy\n",
    "            torch.save(model.state_dict(), \"../model/best_epoch{1}_{0}.pth\".format(epoch, (int(time.time())%1000000)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get dataloader...[ok]\n"
     ]
    }
   ],
   "source": [
    "labels_t = torch.tensor(labels)\n",
    "feat_set_t = torch.tensor(feat_set)\n",
    "dataset = torch.utils.data.TensorDataset(feat_set_t, labels_t)\n",
    "\n",
    "labels_test_t = torch.tensor(labels_test)\n",
    "feat_set_test_t = torch.tensor(feat_set_test)\n",
    "dataset_test = torch.utils.data.TensorDataset(feat_set_test_t, labels_test_t)\n",
    "\n",
    "# split_ratio = 0.25\n",
    "# test_length = int(0.25*len(labels))\n",
    "# train_length = len(labels)-test_length\n",
    "# train_dataset,test_dataset = torch.utils.data.random_split(\n",
    "#     dataset = dataset,\n",
    "#     lengths = [train_length,test_length]\n",
    "# )\n",
    "\n",
    "print('get dataloader...', end='',flush=True)\n",
    "# 划分数据集与训练集\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    dataset=dataset,      # torch TensorDataset format\n",
    "    batch_size=256,      # mini batch size\n",
    "    shuffle=True,               # 要不要打乱数据 (打乱比较好)\n",
    "    num_workers=2,              # 多线程来读数据\n",
    "    drop_last = True,\n",
    "   \n",
    ")\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    dataset=dataset_test,      # torch TensorDataset format\n",
    "    batch_size=256,      # mini batch size\n",
    "    shuffle=True,               # 要不要打乱数据 (打乱比较好)\n",
    "    num_workers=2,              # 多线程来读数据\n",
    "    drop_last = True,\n",
    ")\n",
    "print('[ok]')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "begin to train.....\n",
      "[1,  100] loss :0.982 acc=92\n",
      "[1,  200] loss :0.224 acc=95\n",
      "[1,  300] loss :0.128 acc=95\n",
      "For epoch 1 the test accuracy over the whole test set is 8 %\n",
      "[2,  100] loss :1.148 acc=83\n",
      "[2,  200] loss :0.275 acc=92\n",
      "[2,  300] loss :0.210 acc=95\n",
      "For epoch 2 the test accuracy over the whole test set is 12 %\n",
      "[3,  100] loss :0.125 acc=95\n",
      "[3,  200] loss :0.110 acc=95\n",
      "[3,  300] loss :0.095 acc=97\n",
      "For epoch 3 the test accuracy over the whole test set is 13 %\n",
      "[4,  100] loss :0.075 acc=94\n",
      "[4,  200] loss :0.061 acc=98\n",
      "[4,  300] loss :0.104 acc=99\n",
      "For epoch 4 the test accuracy over the whole test set is 11 %\n",
      "[5,  100] loss :0.040 acc=99\n",
      "[5,  200] loss :0.064 acc=98\n",
      "[5,  300] loss :0.042 acc=98\n",
      "For epoch 5 the test accuracy over the whole test set is 11 %\n",
      "[6,  100] loss :0.036 acc=98\n",
      "[6,  200] loss :0.031 acc=100\n",
      "[6,  300] loss :0.033 acc=98\n",
      "For epoch 6 the test accuracy over the whole test set is 8 %\n",
      "[7,  100] loss :0.045 acc=99\n",
      "[7,  200] loss :0.026 acc=98\n",
      "[7,  300] loss :0.043 acc=98\n",
      "For epoch 7 the test accuracy over the whole test set is 10 %\n",
      "[8,  100] loss :0.020 acc=94\n",
      "[8,  200] loss :0.062 acc=96\n",
      "[8,  300] loss :0.035 acc=99\n",
      "For epoch 8 the test accuracy over the whole test set is 10 %\n",
      "[9,  100] loss :0.035 acc=99\n",
      "[9,  200] loss :0.023 acc=100\n",
      "[9,  300] loss :0.015 acc=99\n",
      "For epoch 9 the test accuracy over the whole test set is 12 %\n",
      "[10,  100] loss :0.052 acc=96\n",
      "[10,  200] loss :0.028 acc=100\n",
      "[10,  300] loss :0.016 acc=98\n",
      "For epoch 10 the test accuracy over the whole test set is 11 %\n"
     ]
    }
   ],
   "source": [
    "print('begin to train.....')\n",
    "# 模型训练\n",
    "train(device, 10, train_loader, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABXj0lEQVR4nO3dd3xT5eIG8CfpSPemi24oLaOsFkrZo1KGynKg6EX0iiIOREFQQcGB4171qojjKqA/t1dwo8h0QBmywbLKpmV2Qmfe3x+nSZM0SZM2aXLS5/v58KFNTs55T3Ka85x3HYUQQoCIiIjIiSgdXQAiIiIiQwwoRERE5HQYUIiIiMjpMKAQERGR02FAISIiIqfDgEJEREROhwGFiIiInA4DChERETkdd0cXoCnUajXOnDkDf39/KBQKRxeHiIiILCCEQGlpKaKjo6FUmq8jkWVAOXPmDGJjYx1dDCIiImqCkydPIiYmxuwysgwo/v7+AKQdDAgIcHBpiIiIyBIlJSWIjY3VnsfNkWVA0TTrBAQEMKAQERHJjCXdM9hJloiIiJwOAwoRERE5HQYUIiIicjoMKEREROR0GFCIiIjI6TCgEBERkdNhQCEiIiKnw4BCRERETocBhYiIiJyO1QFl48aNuO666xAdHQ2FQoGVK1fqPS+EwPz58xEVFQVvb29kZ2fj0KFDestcunQJkyZNQkBAAIKCgnDXXXehrKysWTtCRERErsPqgFJeXo5u3bph8eLFRp9/6aWX8Prrr+Ptt99Gbm4ufH19kZOTg4qKCu0ykyZNwr59+7B69Wp8//332LhxI6ZOndr0vSAiIiKXohBCiCa/WKHAihUrMHbsWABS7Ul0dDQeeeQRPProowCA4uJiREREYNmyZZg4cSIOHDiATp06YevWrcjIyAAArFq1CqNGjcKpU6cQHR3d6HZLSkoQGBiI4uJi3ouHiIhIJqw5f9v0ZoH5+fkoKChAdna29rHAwEBkZmZi06ZNmDhxIjZt2oSgoCBtOAGA7OxsKJVK5ObmYty4cQ3WW1lZicrKSu3vJSUltiy209lw8DwullVifE/zt6Imefly20lEB3mjX/swCCHwf5uPo2NUADISQnDy0hX8sOcsJmXGwd/Lw2bbvFRehTuXbUVCqA96xgfj9j7x2pt0rcs7h5mf78SotCicLrqKK1W1iA70wmsTezRYT02tGsv+PIasdqHoFBWAj3NPIDXSHxkJIQCki5P/yz2BjpH+SA73x6dbT+CH3Wex53QxHhzaHrVCwE2hwMTecYgO8gYAfL/7DC5fqcbSP/IxdUAS4kJ9cPziFXSI8MeEJX9icEob5F8oR8nVagxJDcfz49Jw/OIVbDx4HhN7x+Ku5dvQt10oZmR3wBfbTqJtkDcOFZaie1wwwvw8MW/lXqzLO48AL3c8OCwZ50sr8c7Go2jjr8L5Uun75Mt7s9Crbh80Vuw4hR92n0Va2yDcP7Q9/jxyAR/8no91eecBAC/f0BU3ZsTino+24ed9hQCAZ8Z2wXc7zyAu1AfeHm4oulqNO/rGY+fJYkzOisfF8io89r/dSInwx5yRqbhaXYuPNh3H2eIKLPvzGN69PR3DOkZg2Z/HcLWqBjHBPvBTueOfH27TlivU1xOT+sTjy20ncbZYqpFOjfTH3wWlAID1jw5GQpiv3r4UFFfg5nc34XxpJa5U1aJL2wC8948MnL58FV9tP4VwfxVeX3vY7DEU6O2B4qvVuDE9BoNTwpEY5otRr/8GAFgyqSemffyX3vL3DErC3JEdAQCr9p4FAOw/W4q31h2Gn5c7OkUF4J3b01FRrcabaw9h46ELEEIg0NsDu04VAwDaBnnjdNFVs+Uyp3dCCF6+sSviQ6X3Y/X+QlTW1KKmVmDG5zsbLD93ZCoW/fR3k7dnSteYQOyu2ycA6BYTiPhQX6zeX4gnRndERXUtJvdNgIeb8caMHScu468TRTh6vgwXy6rw/Pg0/H74Ah78dEezyvXrzIH4fvdZfLH1JK7rHo3ZOalwUypwsawSi9cdwQd/5CPQ2wMbZw1BoI/tvo+sZdMalD///BP9+vXDmTNnEBUVpV3upptugkKhwOeff47nn38ey5cvR15ent66wsPDsWDBAkybNq3Bdp5++mksWLCgweOuWoOSMOcHAMCvMwehfbgfAOBM0VWUV9agoloNlYcSpRXViA/1RfHVaqjVAskRjd+6mqwjhED+hXLEh/rCTSmd1AtLKuDt6YYAMyHi+MVyXK2uRWpk/bG57u9zmLJsKwDg+wf643xppfb3Yy+MRtLcH6AWQLfYIPzrhq4oq6yBp7sSwT6e2hO6MWeKruJSeRU6RQVAqay/O+iFskocu1CORT/9je3HL2sff+f2dOR0jsTFskqkP/ur0XX+312ZKK+qQdsgb3SODkBFtRpLNhzB62ukvmTL7+yNyR9sAQAcem4kzhRdxYlLV3D7+9Jj7koFatSmv1Y+vLM3Ll+pwkOf7TS5jDHTBrfDkvVHGjw+e0QKXlqVZ+QVltk1fzh+3l+Avu1CcbCwFHcuqw8Fo7tG4YfdZxu85tWbu+Hhz3dZtP5nxnZB7tGL+L5uPeN7tMUPe86iskatt9w9g5LwzoajTd4PQDqWyiprUFZRg8hAL+13iS43pQK1Zj4fW/ht9hAE+Xgg7elfjD5/U0YMvt11BhXVaqPP24LKXYkfHhyAw+fKcO//bbfbdpprZJdITB2YhPwL5cjpHAlflVRvcPLSFQx4aZ3esqPTovDDnobHY3M9Py4Nt2bGIfuVDTh8rr4/6LgebfHqzd1tui2H1aDYy9y5czFz5kzt7yUlJYiNjXVgiVpGQXEF2of74cTFKxj48jqzy756czeM68EaF1v6avspzPpqN67tGoU3b+2Ji2WVyHx+DQDpRGDMsQvlGPyv9QCApXf0wpDUcBy7UK4NIwBw7Ru/671GCAHN+WLXySJc8+pGved/mz0EsSE+Dbale1zc3icez4ztAgBQqwUyTISPo+fLUVOrNhlOAOC293O1P780oSte/fWg9oodAPLP13+B3blsK347dAGDOrTRPmYunADAP+rCjbWMhRMAzQonANBtofGTKACj4QSAxeEEAPafKdaGEwD4esdpo8s1N5xo9HxmNapq1Hh+XJrR5+0dTgCp5k7lbrqL4xfbTtm9DJU1amS/ssHu22mun/YW4Ke9BXW/7cKxF0bjXGlFg3ACwC7hBAAeX7EHvRND9MKJZnu2DijWsGlAiYyMBAAUFhbq1aAUFhaie/fu2mXOnTun97qamhpcunRJ+3pDKpUKKpXKlkV1uF/2FeCln/Pwn4nd0Tk60OgyNWrp6qKxcAIA//0t3yUCyo97zuKV1Qex+NaeSIlsvFbolV/ysOHQBXx2dx94e7pZvb0dJy5j3Ft/on24H36dOUjvucXrpKrv73efxR19L+GGtzdpnxNCYNi/N+DohXLtYxnxwRjdtf64/2TLCQxJDcfGQ+fNliFx7o9mn8/Nv6QXUJ7+dh8+yT2Bqtr6q8+PNh/HR5uPm10PALy46m+8uMryquzZ/9vd4DFNExEA/HboAgCpWZKM+3TLyRbb1sR3N6Gqrmbm8RV7Wmy7hpy5xsLZrTlQ2CAotARjYe6u/oktXg5dNp0HJTExEZGRkVizZo32sZKSEuTm5iIrKwsAkJWVhaKiImzfXn8Ar127Fmq1GpmZmbYsjlOb+tF2HD5XhtGv/47//lZ/5aTWubq5UFZltIrWGHNNDnJy38d/4fC5MovbWF9fexi7ThZhhYmrUkPllTX41895eGX1QXzwez7GvfUnAODwuTK8/PPfKL5aDQA4X1qJYxevaF+nG04A4OZ3NuuFEwDYdvwyzui0m6/eX4hR//kNpRU1FpXNFM+6K1G1WuCNNYew7M9jeuGkpT317T6HbZvM23z0kqOLAAA4W1yhV+tGlrtr+TaL+8N0igrAsRdG46nrOtm8HP/9RwYeG5Fq8/Vaw+oalLKyMhw+XN+pKj8/Hzt37kRISAji4uIwY8YMPPvss0hOTkZiYiLmzZuH6OhobT+Vjh07YsSIEbj77rvx9ttvo7q6Gvfffz8mTpxo0QgeV6DpoKfx7A8H0CMuGF1j9GtSHv3S8mrk2BDT/RTkKK+wFAfOlsDDTQk3pQKJBp3/jp4vg8qjvsbkbHF9MBBCYM/pYiSH+2trVYQQ2H2qGAu/36/XJ0PX4nVHsPbv8/jXjV0x+6uGNQe6thwzfiI4Y/ClvP9sCfafbV6n7gt1x8u/V+dh8TrjzRxErcmPDw6Ar8oND366Q9ux1p6eGdsF6/4+h7V/69f+7356OLrW9bN55aZuGJISjh7PrLZ7edr4q/DN9H6ICvQCAEzpl4gF3+236TayO0XYdH1NYXUNyrZt29CjRw/06CH19J85cyZ69OiB+fPnAwBmz56NBx54AFOnTkWvXr1QVlaGVatWwcvLS7uOjz/+GKmpqRg2bBhGjRqF/v37491337XRLjm/Xs81bP+fsORPPP3tvia3Dze9q7PzGvmf35D9ygYM+dd6XCirD3XnSysx9N8b0O+FtdrH3tAZifD51pO4/s0/cLtOX4ovt5/CmMV/mAwnGgfOlmD0679j35mmhQpTfRaaY+H3+5FXUMpwQnaT+/iwBo+lxwc7oCTmzcpJwdyRqegUHYD4UF+8fXu61esY2SUST47uaPL5N29tOIqtR2wQ3rilB14z6I8R4OWBDbMG461JPTGuR1sEetfXZHvq9MFJsfEghjA/FaKDvPWbW2cPwYSeMfjftCyjr/nftL4Wrz+5bnCGo1kdUAYPHgwhRIN/y5YtAyC1Ty9cuBAFBQWoqKjAr7/+ig4dOuitIyQkBJ988glKS0tRXFyMDz74AH5+zvGG2MOmIxfR+7lf8fO+ArPLfZx7AqnzVjVpG7UtnFC+2HYSmc//iv1WnsgvlUvNVglzfsDJS1e0P1fW1GKriVoJAMh49lfM/Vqq1ThUWGp0mfs+3o5atcCcr6W2923HLyNhzg+Y/vFf+GJry/UDsIeN7ONhE/4qd+R0jsD/pmVh+pB2Di1LqK+nRcsNN3Elu+D6zhZva/aIFLPPRwR4YVaO/jLvT87AC+OljrZpbY33k7MFXyv6jk0f0h73DKr/3KICvfHA0PZWbe+G9Bjc1ife6HP/m5aFa7vq1+Tf0jsOnaMD4Ktyx9gebfHLwwMBAFMHJgEA4kN9MSotCgqFAkqlArNHpOCegUmY2Kt+IMd/J2cgxMLP2xLGBt/Ghvjg3zd1Q3p8SIPnnrquE9Ljg/HN9H4Wrd+wxtpReC+eFnDLe5txrrQS93xkv45jMWaGotrD7K92o7CkEjO/2GnV6z74PV/7s24v9V/2FeJGg34ehj7dchLHL5abfP7HPQVo93jDDqc/7DmLbY3UnDg73SHE9nJH3wS7b0PXgOQw7fBtQ7dmxtl0W6tmDMCxF0Zjz4IcvHN7BtLjQzArJxWb5zasOTDmjzlDbVoeAHjz1p5GH/98ah/c0TcBvp5u+P2xIXj3H/VzRvmr3HHshdE49sJobfW+JSb1Nn5CBuqv7tu1qb9IvGdQEoJ8PDGxdxyOvTAawzqGm11/j7ggi8tiaM/TOTj2wmg8MLQ9vDyUuK6bfkBY/+hgeHkocfcA4x02Hxmegu/u72/x9hQK6I0wCvT2QFSgF4amhmtP7tOHtIOPpxvWPDIIi8an6dVUdIjwx7EXRuPxUcZrYe4b3B5zR3XUO7YVCqlmxsOtaX/Hec+OaNLrAGnE4ZR+0nvXLTYIo9Ii0cZfhS/vNV7T4umuxJ0O7hyrIYthxnJVqxb4dMuJFtlWaWXzOmJao0Cnn0W1QWfNv05cRkSAF9qaCExqEzU9D1jYKXbQy+ub9WUoV5bkk2+m98OYxX80af1dYwLx9PWdsezPY016vaV0Jxa7vls0lt7RCwLS34qnmxK1QqBWLeDl4YZnx3RBkpHAaalrOkVg9f7Cuu0an28h0sKTvAKAp5uyQefkpDBfHL1QjidHd8SzPxxodD1jukfjm51nAABZ7UKNLpOZFIrMpFA8MbqjdgKvfu1D8cfhi5ikc+Wve9I059WbuyHQxwO9E0OwJb9hLaUm/KbqjJrTTLSmYa7luUdcEP53b98mf1aa7T8yPAUPDkvGjhNF+G6X9B55uimREOaLPU/nmJzMDADSYgJx6LmRUABo/8RPZreXHO4PhUKB7rFB2HmyCHf1T8R9g9vpBYpZOamYkd3B7DYbMyQlHEv/OAYACPLxREywD/YvHIHkRspnaOOsIVC5Wz9C0ZTFt/ZEjVrgSmWt0ef3PD3cpttrDgYUG6lViwZXgx/8no/nfjzQYDl7WPrHMTx1neVVvs3RZ9Eao4//XVCC8XWjYvIXjUJljRoqd6X2i7RWLaC08EvVnB0nipq9Dmc1qEMbo0N2LXnXQv2MVyHfmhmHT3JNB+W3b0vXniwfHNq+0ZlFm+OzqX2w5sA5hPh6YlCHNtqTk6a/sxKK+p/NpLLPp/bBJ1tOaE/2naICGnRG7hDhpw0o1rghPQYPX9NBr4+TAPDbY0O08+AAwM8zBiIiQIUdJ4swMLkN/v3LQVytNv6lr/HKTd0xKTPeotoP3ZPj27elY+uxS+jfvn6+mRBfy0buBflIx0WHCD+jAaVDhFRzkhDmi/9Ny0Kob8MpHSIDTJfX38uj0Ro+3WBqjoebEr0TQ/DJ3Zk4ffmqdn4dS4JCY8use3QwLpVXaYfsf3hXb2w/dhn9k8PgbuS1zQknADCwQxvMv7YT4kOlWYGbuk4flfQH8evMgch+RZojqX0jfUQSQn30RiHqUigU8HBTwN3N+EhAZwknAJt4bOJSeRV6PVffR0LDMJwAMNoEIWe6cWviu5u1PyfO/RGp81Zp5/j4aPNxdH5qFf46Ie+mFnsL91chIbThpGyWNPHEBOu/LtjHA7dmxuGxHPNDBYd3itB27ntgWLIVpTUvxNcTGQYdLYN8PDEhPQZDUsOb1WyVmRSK127ujg4Rfgjy8cA399e3rXu6KXH/kPZ45JoU3D0gEW/c0rDTozlZSaENagDVaoGIAC9464wcS4n0R5CPJ4akhMNNqUBj2XvLE8PgplSgd2KI9iS5ZFJP+HvVXydq+nwY8vfywNDUCL2Olz3jgjF9SDv868ZuJrc5dWASBted5GeZOA50L2zS40MaTJcPADdmmJ5jybORk+7ADm3w7f39GwxZvX9Ie7x8Q1ejr+nbLgw3ZsQi3EwwMsVU/4nEMF+9jr8BXh4Ykhre7CBizp39EzGsY9NHw9zeJx5hflJgbB/uj5XT++GW3nGN9j/y8Wy87kEzY60zY0CxgY83H8el8ip8uuUkco9etPv22vi3zKR1b6w5hDfXHrJ4+aIr1UYff+6H/Zi3ci8qqtX484j93x856WnQXCUAvH5LD4QZ1IY0Vp0/OEU6CT1bN5ssIPXaf35cmtl7aXi4KfSCgqkv6wAvy7/MIgJUeDi7A/6adw2+mtYXH9whdRBcOqWXxevQ6BYb1OAxzYlOoVDgl4cHYef84Xrl/kdWPB7NSYFSqcATozs16NNgTmKYL0amSRNG/mdid+3jmsBiqonSEuH+DU+2I9OikPv4MCSE+uCW3nGY2NvyvjcKhQKzclJxQ7rp8PD4qI7aY0d3hIkuSzpvGjsuxvVoizb+KjxhZkQMIM2o7OmuxLTB7dClbX1T26M5Kbgxw/YzgjtLB09L3NkvESG+nvjort5Gn3/UoONy99ggLBqfhlA/8+cAS2vqnxvXRe/38T3bWvS6luL8EUpmbn53s8lp0G1hdNcovHlLD2Q8+ysullfZbTvFV6rx79UHAQC3ZyVov9x0O7kC0K9CMeG93/IbX6gVGt+zLSoMmgSEALrGBGHrE9l6M8zuOllkdl3jekhfLLf1icekug6mlvRROPTcKIvK+sDQZKM1gsZsnjtMb9tDUyOw/clsi/tM6Pr3jd20M1zmL5LKamo9wT4euHylGqN0ZvO11tpHBmnXP6Z7W1xfF240j5kLKP8ckKS9X5E1fDzdse7RwU16f5rL0pFExjx9fWcEeLk3Wm7dpu+be8Vh7+m96JVgvyHMxirmUi2YldoR5l/XCfOu7WjyPWysdsoUzSzkjTE8nGcM62B8QQdhQLGBlvpe+ereLKTFBEKhUGDtI4Nx/FI5dp0qxryVe/XuhWILpZX1tSGaqbMPFZZi4ffGJwNqxj0nWy8hdd78cU/D4eeGX1hfbTd+75LPp/aBh7sSPXRqGmx9ovvftL7oERtkUUDxdFMa3X5Ty9Q+3A8/PjgAbfxVja5jzSODcfryVaTFNH1IrOE2DH83d2H60LBkDOoQhglLzI9Gs2S79vL8uDTtFPh9kkLw9m3WzyPycHYHXNstymSNjDmTesehU1QAOkbZMzA0fC/facJ8KfbSPtwPh8+VaUdPaT77qECvBrPvNnXUj6U1KNFB+rV6TbldiD2xiUdGMhJCtB2YAn080DUmCJ51B7C7DYehvv97Pvq/WD8EWBM+DhY2vD+E5s/gyZV7bbZ9Zzb/2k42a2LzdFdq51LQfcwavip39IwLbtIJ7pO7zd9aoltsEF6/pQfS44OhVCrw/QP9oXJXmr0/x48PWT7c01KdogMses9DfD2bFE6sqdb28TD9Be6mVBidg6IlGKuRyDYyNNhPp6muW0yQtgOtNdoGe+sNSbaGUqlAenywRX0kmsrYV6G3mc+tpS2/szemDkyyqMnT1DD8xlja0XVIiv4x0lLdByzFgNIMFdW1uOejbQ2ubu/9aDsGW3CDP1uwpOrZWs8Y1JJsPHQBU5Zu0bvPjIYmvHxsZpSIK7mzfyKeMDH/gSVu6R2Hedd2QvtwP8y8pkODO74+nG27TqqN6dsuzOjj/+yfiG4xgfh8ah9tEwcAdGkbiLxnR+KR4cargUd0jkT7cOesSjfnubFpSGsbiHsHNT5x27I7eyMh1Acf3JFhcpnldxrvT2BPyUZmKo02MtQ/p7NOh80mXtNY87KmNlE0h7GRgs504m0b5I3HR3Vs8Pl0jNIfCh/m59nkmrVXb+6OxDBfo7Pi6lIoFHj39nQkhPpgpYWTuLUkNvE0w4ebjuHnfQ2HMa5qZMbYpog3MrIDqP+ysGcDi+aeQOvyGg5/lWPDzvBOEegWG4SXf87TPhbm54kLZcb79PxnYnc89NlODKnriGrsi1+335G5GzzOGZmKQG8Po7UQ/3dXZpNGLdjak9eav/GYqaHiSple7nh7uuG7Byyr+UmPD8b6WUPMLjOoQxtMG9wOS9bX35rgRjMdWe3F2Keke2VtiyH/ujRNF87IEf17rGVYWfLjQwOavK5O0QFY9+hgi5Yd3jkSwztHNnlb9sSA0gyaO982h6e7UtvHIyJAhcIS/RsJRgSo8PnULESYOHEptTUozS4KAOv7kmjKLicdjFxtmtvtMd3bonN0AOJCmjc64I85Q82229vzOzQywAsFJba5u6ypE5uiqZfkrcALE4wPp7WnDo10DNXMf2ItU8fp7X3iG9zpWjjgEsaWtcktyfA73Nior9ZGptc8zsEWX8gbZg3W/pxhpP1aqVAgIczXZOclzVWrrTqpWtuXxB63VLd3e7GxOR10v9Tevq0nnh3bBU+O7oiv75NusNU+3N9k/xA/C+YTmD0ixeTsuhqW9j/RvWeKpW3UX03LwqyclCZ1bDRkcpvMJ1qGf45N7UtgKcO1x4Z4Y2Iv40OWV9zXF0+O7ogx3Zo2pNTUcerl4RynEzvNhWl3HGjQEGtQmuDkpSuY8fnORu+M25jfZg9BVKA3Xhifhi+2ncTD13TAD3v074bb2BebJiTZ6thuSl+SIf9ab5uN1/H2dGt0Rs7mMDang+6X2sAObSzuxNc1JhDPjOnS6HLmOpbePSAR+ReuID1Ov6PjqLRIoyN8pg9pj/OllThXWmHx8MmYYB9MH9IewzqG47GvdmPmcPM3jzPH1CFp6yYDspzhn//dA5JMfnf0iAtGD4NjzRIzspOx/fhl5JhoDhjTvS0e+98e/XI54Jwr1xO9XIOVPTGgNMFj/9vd7HACQDub5MS6CZouG5nXxPAOo4Y05wRHVmvmXzB9A7+msOZi09jQvMYY++KOCFBpm+y8rJjq+VsjNymb0i9Bew8OQOrzYq5X/ROjjff5yIgPMRpQAGkOiqZIjQzAN1bcWM0YU+35jCf1HJ3V7PF1MCPb/BwZXkZqPudfZ74/kz3cN6Q91vx9rsW321zMJw0xoDTBJTtNkKb7paa5y2pjNzNT2mEUj6NZeiW+/cls3LV8mzagZCWF4s1beyD92V8bXb/hJoakhuOraX2hVCiafefg+dd2wj0D20GpAGrUwux9TMxxNzIHwq75w5tVNntKj7f+qtxVdYlu+lwsTWF4pDjL98E/shJafJvp8cF4+rpOePo7aTSiszQ9NSYzMQQbjdyHqzWTxyfnZKy94d/PMwY2aTuW3GlVc6J1ku8jiz1qYqgqYNmQwBGdIxHqp9LepwIAltzWE6F+Kvwjy/St5QGphibAq74vxjNju+Dh7A4I8PKwqD9JY194CoUCkYFeCA/wQnSQd5MDj2FQiwr0MjttvaNpZrAlqXmuJRn++cvhPiv2NLlvAp4fl4Zbesdi9cODHF0ci9w9IKnxhVqZ1n0UW0kIgbs/3I5DVg6lS7Gwn0BTOt1qTmLNCSiXy6sw6b+5De4Ga0+RgcY7jHaI8MMTozvi1vdyG3m9FN6eHdsFlTW1mNIvQTvp1GMjUvHhpuMmXysgdZTdePA8BiSH4fY+5gONobS2gbghPabRTq/NZTj53uS+CXbdnjUGp7TB+rph5wOSwzD/2k5G7wjbWuk2gzXWTGtrOZ0jMLa74++pYosO2U2lUChwa2YcAPmEZk93JWKCvXHqcsP5plorfqNY4fjFK/j1gHW3b++TZPnMkv5e7gj09oCvp1uDm8WZorSyD8qR82V4adXfev1dPtlywqbh5Plxxu/IqstPZbxPxvI7ezcYXtfJYAIjXZGBXvjorkwMTa2fgMpX5Y5/mumUKoQ0H8S7/8jA7U2oglYoFPjXjd3w8DX2vW+FYV+Zn+0wv44tfHRXptGJwkhi7yAL6DfxvHN7htUzEtvD9VbcpJEkA5KNT57YWrEGxQo1TehmPSzV8lttK5UKbH0iGwLCiqvRuhoUC5ce+Z/fUFWjxtHz5Zg1IgX+KnfsPlVkcRkb8+Tojrg1Mw55BSVYbqYWI8DE1ZW/lweuVOmP4DHWFyOjkZuNPXltJ/zX8MaGdXxNhCNnYzizpOGss44ktyZFV9c7McTpZnN2xBwo5FoYUKzQlOFrtVa+xtorH2trUDQTq63aV2CXGW815Zg7qqPJgLJyej+95gvdkTh+KndY0v9+dFrjd6x1VyqMhkp73gfElgwDirsTTdXqLJ0w5SDA2/7H2/XdouGuVKJrM26UaGscNtsUHAunSx7f1E6iKX9w1naotZbCxjPJNpemH42xIYca3WODIITAlH4JaBvkjRU7TusNFTbsHGp4Lgz28ZDF1NW21gp3WdaeH5eGPaeLMbhDw5v22ZpCocDoro2H9pbEDGs9/o3rc55LMhloSpWlqYDyn4ndm1kaifZ4dpJvgxtM3HPEcH8VCgWeuq4z/jkgqUEgMfwjtcXVuuZ266/c1K3Z62opzvxd5SSHm1O7NTMOi8anNXvYulxNs+Dmi6SvdR4ppjGgWOFqlfWzm/ZO1O8k2yHCD4eeG4kxNupl70yJO9TXE8G+1t++3XAfDEczCQHcMzBJZ3nLdloz7HVQhzbI6RyJQ8+NxPieLX/TNltxppla2cRDjYkzcYNTMs2J/sSdApt4rDD36z2NL2SgT1Ko3u/J4f5Gp1p3BeZuM28uVBg+Z2zRR4an4J2NR60qz+OjO2JwSrg2JMrtfW8Q3Jzoy4v5hMj2YoIZ6nQxoFjh74JSs89/cEcGNuSdN9o5dOX0fvh86wk82ox7oJhjyfnirfWH7bJtDcPak6Q2vjh6XpoG31wtt+F8H4YnYgH9zsOW3nhN5e6GIan2b/9vKU6UTzhCg8gOpvRLwJmiqxjqQt9bzSGvS0onNzQ1AgvGdMH7kzMQ4uuJpVN6aZ/rHhuEReO7ItSv8VlSrWHNVfVLq/Jsum1DHgbB4ct7shAd6IV7B7VDdscItGvja7SPyqLxaQjzU2FB3f1lDGtUNKOnXr25G0J9PfH2bel22gPn0rBmyXkiCmtQyBje7qB5VO5uWDimCwanMKAArEGxi2EdI7D9yWynOqHY2u194vHRZv2aIsOajVA/Ff6YM1T7Pvw6c5DR96RDhD+2PjGsfkSSQcfiW3pLfUnG9YjB2O5tXfp9NceZ9npCegxy8y+hc7TpSfSo9Zk7MhU3vL0J3mZG8RFZigHFQtW1aquWl/tJdOGYzpj/zT6TzxtrZjH2mO77YGk/FN1RD1/dm4WeOreGl/v72hzOtO83psegfbgfUjiDLOnISAjB9w/0b5HZc8n1MaBYqLGA8v7kjBYqiXG2rnLXDQXG2HNW07ZB3njquk4I8fVERoLltwpwdc50V1aFQtHoMUKtU5e2zjNZHMkbA4oF3t14BNuPXzb5/NI7ejmsM2ZTbjBoCW9P81W09wxq12BUjS1vDjaln+l76bRWc0d1dHQRiIhajPNckjmx53/8Gz/vM32TQDmMFCnQmanVEu5KBab0SzD63N4FOQj2aRhGnKkJwhWx2pyIWhPWoLQSYxf/YdXySoUC80Z3wtI/jgEAJvaKxaycFNSoBfxUPGyIiMi+eKZxEY3NS1FQYmUNiptCr7Nq57aBNh8iTUREZAqbeJpJMwTWYezUqqIZkTNnZCoyE0Nwg4yniCciIvlhDUozPTq8g6OLYBfuSim73juoHe7lTb+IiKiFsQalGbY8Mcxlmz0snU5eY3wP29z8kIiICGBAaZYwX+cJJ7aeB8Xw/jiNsTbQEBERmcOA0gxKJzgp26sE1gaOg4Xmb6RIRERkDQYUMsraGpRdp4rtVBIiImqNGFBagaoa6+4jBLDJhoiIHIsBpRFCJveVN1fMhz/fafX6OCssERE5EgNKI9QmTvxPjnaO+6JYEiR+2HO2BUpCRERkOwwojVAbqZq4OSMW/xyQ5IDS2E9yuB8eG5Hq6GIQEREBYEBpVK2RKhQ53BzQWr88PBDXdYtydDGIiIgAMKA0an3euQaP5XSOcEBJzGtKTxndGWIVCgVign3w9X19se7RwRa9/ppOzvc+EBGRa2BAacSvBxoGFGfqQNqckvh7NbzTQc+4YCSG+Vr0+uRwv2ZsnYiIyDQGlEbU1OoP0Z1/bScHlcT2PNyaF7SsnSuFiIjIUgwojag26IPSJynUQSWxPQ+35n38zjCTLhERuSYGlEYY1qC4N7PWwV6MzddytarW7DwuzZ2MTWG3ifaJiKi1a9gJgfT8vK9Q73dnm2HVVHeYwpIKZD6/BoM6tGnxbRMRETUXa1Cs5CaTs/LKHacBABsOnje5THM7+8rjnXAN9w1u1/hCREQuhAHFSs5Wg2KKJdmjuXsik6zmEtLjgx1dBCKiFsWA0oiYYG+9331V8mgVs6R/iJIJg4iInJQ8zrYOpNvH9OUbuiLE19NxhTGiKR1VHx3eAd1ig3Dq8tVmbdvHk4dPS2GWJKLWhjUoZpwrqcDpovqT+I0ZsQ4sjXXMndDuH5qMAclt0NzWqlt6xzVvBURERCYwoJjxwqq/HV0Eu2ruMGFvTzcblYRM0cz22z2WfVCIqHVhHb0ZZRU1ji5CozQ1JWamOzHz4uZvf2CHNth48Dx6xgU1f2XUwNYnsnG1qhbBTta0SERkbzavQamtrcW8efOQmJgIb29vtGvXDs8884zehGFCCMyfPx9RUVHw9vZGdnY2Dh06ZOuiNJuzTspmCVMdYFfc11f7s78NOvy+PrE7Fo7pjPf+kdHsdVFDXh5uDCdE1CrZvAblxRdfxJIlS7B8+XJ07twZ27Ztw5QpUxAYGIgHH3wQAPDSSy/h9ddfx/Lly5GYmIh58+YhJycH+/fvh5eXl62L1GTuSvm2gBnLJ9GBXugRV99UMLxzJEZ3jUKP2KAmbyfIxxP/yEpo8uuJiIiMsXlA+fPPPzFmzBiMHj0aAJCQkIBPP/0UW7ZsASDVnrz22mt48sknMWbMGADAhx9+iIiICKxcuRITJ060dZGaTCZTnljMsBXITanA4lt7OqQsRERE5ti8iqBv375Ys2YNDh48CADYtWsXfv/9d4wcORIAkJ+fj4KCAmRnZ2tfExgYiMzMTGzatMnoOisrK1FSUqL3ryU0pVtHS9NkKGFQ2tNGhhDbc9p7IiIiW7J5DcqcOXNQUlKC1NRUuLm5oba2Fs899xwmTZoEACgoKAAARERE6L0uIiJC+5yhRYsWYcGCBbYuaqOa1PHUCVytqsV/f89v8PjtWfEOKA0REZH1bF6D8sUXX+Djjz/GJ598gr/++gvLly/Hv/71LyxfvrzJ65w7dy6Ki4u1/06ePGnDEpv27a4z2p+jAp2nb0xjLpRVGn3c002+fWqIiKh1sXkNyqxZszBnzhxtX5K0tDQcP34cixYtwuTJkxEZGQkAKCwsRFRUlPZ1hYWF6N69u9F1qlQqqFQqWxfVKjemxzh0+7bQ3JsDEhERtRSbX1JfuXIFSoPRL25ublCr1QCAxMREREZGYs2aNdrnS0pKkJubi6ysLFsXx2be/e2oo4tgnJF5UEw1Tblap18iInJdNq9Bue666/Dcc88hLi4OnTt3xo4dO/DKK6/gzjvvBCBdxc+YMQPPPvsskpOTtcOMo6OjMXbsWFsXx2YqqtWOLoLFDDvMavDmgEREJBc2DyhvvPEG5s2bh/vuuw/nzp1DdHQ07rnnHsyfP1+7zOzZs1FeXo6pU6eiqKgI/fv3x6pVq5xqDhQ5M1WDUl0rn5BFREStm80Dir+/P1577TW89tprJpdRKBRYuHAhFi5caOvN282bt/ZwdBEsZmrw0dXq2hYtBxERUVNxWIeFBiQ75xwimhv+6YYSYaIKpW2QdwuUiIiIqPkYUCwkpw6mpmpQQv0cOxKKiIjIUgwoFpJTB1O5TjBHRESkwYBiITkFFHlM0k9ERGQaA4oJhv04nDWfKLTzoNSXlzUoREQkdwwoJpRU1Oj97qwBxRg1AwoREckcA4oJVTX6c4bIqYln/jd7HV0EIiKiZmFAMaHWoBpCTgElN/+So4tARETULAwoJhjOuuqsw4w1xWKrDhERuRIGFBMMa1DkcifgQ4Wlji4CERFRszGgmFCjlud9a84WVzi6CERERM3GgGJCda08G03kWWoiIiJ9DCgmGDbxOCtt01NdcdWcBIWIiFwAA4oJhp1kZYP5hIiIXAADigk1OjUo393f34ElsY5gQiEiIhfAgGJCTV0flPbhfkiLCXRwaSzHFh4iInIFDCgmaEbxuDvrBCh1DLqgmAwoT1/XqUXKQ0REZAsMKCZUVksBxcNNXm+RqQqUO/oltmg5iIiImkNeZ98WdOBsCQD5dZY1vAszERGRHDGgmOCrcgcAlBrc1djZaKe6rwsmjCdEROQKGFBM0Jzo0+ODHVoOa7EGhYiIXAEDigmaE71MbsGjxXxCRESugAHFBM2JXimzhMJ8QkREroABxQTNlPHOHk8MhxkTERG5AgYUEzQnfIXcalCYVIiIyAUwoBhxvrQSn+SeACC/Pii8WSAREbkCBhQjbnlvM05cugIAcPKJZBswFlCmDkxyQEmIiIiajgHFiMPnyrQ/XyircmBJLCElKE0ueeiznQ2WkFtHXyIiIgaURvx26Lyji9BszCdERCQ3DCiNcIUuHcwnREQkNwwojXCFTqesQSEiIrlhQGmE2snzSf08KE5eUCIiIiswoLQCCjbyEBGRzDCgtAJs4iEiIrlhQCEiIiKnw4Aic5rKEXN9eVmBQkREcsOA0hqwjYeIiGSGAaUVYDwhIiK5YUBpBViBQkREcsOA0oiYYG9HF8EshUL/XjxGl2EdChERyQwDSiOW39nb0UUgIiJqdRhQGpEU5uvoIjQbm3iIiEhuGFAaoZDR2b20otro4/LZAyIiIgkDiszpho8HP91hfBkmFCIikhkGFBeyLu+80cflVAtEREQEMKA0IMwNhyEiIqIWwYBiQG75xFTlSN92oS1bECIiIhtiQDGglltCqWNY8/P8uDTtz2zhISIiuWFAMSDPeNKQm7I+lXCiNiIikhsGFAMyrUBpQKkbUJhPiIhIZhhQDMiticdU7YhOPmH9CRERyQ4DioswjFW6wYU1KEREJDcMKAZkVoFikpKhhIiIZIwBxYBwkW6yupOzsZMsERHJDQOKAbXM8omp5hvdx9nEQ0REcsOAYkB3PpEHhyU7sCTWMWyaUjKVEBGRjNkloJw+fRq33XYbQkND4e3tjbS0NGzbtk37vBAC8+fPR1RUFLy9vZGdnY1Dhw7ZoyhW0z3Pu8n4JM8+KEREJGc2DyiXL19Gv3794OHhgZ9++gn79+/Hv//9bwQHB2uXeemll/D666/j7bffRm5uLnx9fZGTk4OKigpbF8dqujURcj7J6/VBkXHQIiKi1snd1it88cUXERsbi6VLl2ofS0xM1P4shMBrr72GJ598EmPGjAEAfPjhh4iIiMDKlSsxceJEWxfJKrpNPHI+rys4DwoREcmYzWtQvv32W2RkZODGG29EeHg4evTogffee0/7fH5+PgoKCpCdna19LDAwEJmZmdi0aZPRdVZWVqKkpETvn73o1qDIqebBcPSRUsF5UIiISL5sHlCOHj2KJUuWIDk5GT///DOmTZuGBx98EMuXLwcAFBQUAAAiIiL0XhcREaF9ztCiRYsQGBio/RcbG2vrYmvpnublfGLXbZ5q469yXEGIiIiawOZNPGq1GhkZGXj++ecBAD169MDevXvx9ttvY/LkyU1a59y5czFz5kzt7yUlJXYLKbpNPHLuJKuAAm9N6ontxy9jVJcoRxeHiIjIKjavQYmKikKnTp30HuvYsSNOnDgBAIiMjAQAFBYW6i1TWFiofc6QSqVCQECA3j970Z0HZWLvOLttx1bMzYMyKi0K867tpHfjQCIiIjmweUDp168f8vLy9B47ePAg4uPjAUgdZiMjI7FmzRrt8yUlJcjNzUVWVpati2M13b4cgd4eDiyJdTgPChERuRKbN/E8/PDD6Nu3L55//nncdNNN2LJlC9599128++67AKSOpzNmzMCzzz6L5ORkJCYmYt68eYiOjsbYsWNtXRzr1Z3o5V7pIPfyExFR62bzgNKrVy+sWLECc+fOxcKFC5GYmIjXXnsNkyZN0i4ze/ZslJeXY+rUqSgqKkL//v2xatUqeHl52bo4VtNURMhpBI8xci8/ERG1bjYPKABw7bXX4tprrzX5vEKhwMKFC7Fw4UJ7bL5Z1HVtJXI5vZu6ESBrUIiISM54Lx4DQtvEI68zvOE9DlmDQkREcsaAYkB7ouf5nYiIyGEYUAwImTXxEBERuSIGFAOaJh65tJDIpZxERETWYEAxINs+KIadUIiIiGSMAcWAZqI2ecUT4EJZpaOLQEREZDMMKAbU2iYeuUUUIiIi18GAYkDbSVYm+UQu5SQiIrIGA4oB7UyyDi0FERFR68aAYkCwiYeIiMjhGFAMyK6Jh3U9RETkghhQDGiaeOQ2zJiIiMiVMKAY0DbxOLYYzXJt1yhHF4GIiKhZGFAMaOdBkXFCuXtAkqOLQERE1CwMKAbUas1P8kgoxoIUm6eIiEjuGFAMaGpQlDI+xyv5qRIRkczxVGZAbjcLNIY1KEREJHcMKCbIefguAwoREckdA4oBtZBXE4+xYsql7ERERKYwoBhwhZlklUwoREQkcwwoBkTjizg9NvEQEZHcMaAYkNtU98awAoWIiOSOAcWAuq4KRS61EMaKKecOvkRERAADihHyr0GRc9mJiIgABpQGXOFePERERHLHgGJA00mWo3iIiIgchwHFgFottyaehgWVTdGJiIhMYEAxoK1BcWgpmkc+4YqIiMg4BhQDrjBRG0fxEBGR3DGgGBAym+reGBlnKyIiIgAMKA3U1gWUmlp5zClrfB4UIiIieWNAMfDuxqMAgKMXyh1ckqbzcOPHSkRE8sYzmYHfDl1wdBGaLdjX09FFICIiahYGFBczoWeMo4tARETUbAwoMmfY34QdZImIyBUwoLgY5hMiInIFDCguhjUoRETkChhQXAwnaSMiIlfAgCJzhjPesgaFiIhcAQOKi2FAISIiV8CA4nKYUIiISP4YUGSOw4yJiMgVMaC4GOYTIiJyBQwoLoY1KERE5AoYUFwMhxkTEZErYEAxMDQ1HACQGObr4JJYxrDGhDUoRETkChhQDITU3Qn4poxYB5ekaZhPiIjIFTCgGBBC+l+uNRGGE7cRERHJEQOKAQEpocj1NK9kQCEiIhfAgGLg679OAwCKr1Y7uCSWMewU6+7GgEJERPLHgGLCe78ddXQRmoQ1KERE5AoYUEyorhWOLkKTuPETJSIiF8DTmYtxYw0KERG5AAYUmTPMI25KfqRERCR/PJu5mKraWkcXgYiIqNkYUFzM4nVHHF0EIiKiZrN7QHnhhRegUCgwY8YM7WMVFRWYPn06QkND4efnhwkTJqCwsNDeRSEiIiKZsGtA2bp1K9555x107dpV7/GHH34Y3333Hb788kts2LABZ86cwfjx4+1ZlFZjwfWdHV0EIiKiZrNbQCkrK8OkSZPw3nvvITg4WPt4cXEx3n//fbzyyisYOnQo0tPTsXTpUvz555/YvHmzvYrTakQEeDm6CERERM1mt4Ayffp0jB49GtnZ2XqPb9++HdXV1XqPp6amIi4uDps2bTK6rsrKSpSUlOj9I+OUHGVMREQuwN0eK/3ss8/w119/YevWrQ2eKygogKenJ4KCgvQej4iIQEFBgdH1LVq0CAsWLLBHUV0ObxZIRESuwOY1KCdPnsRDDz2Ejz/+GF5etmlumDt3LoqLi7X/Tp48aZP1ugLDPMIaFCIicgU2Dyjbt2/HuXPn0LNnT7i7u8Pd3R0bNmzA66+/Dnd3d0RERKCqqgpFRUV6ryssLERkZKTRdapUKgQEBOj9sze5VkTwXjxEROQKbN7EM2zYMOzZs0fvsSlTpiA1NRWPPfYYYmNj4eHhgTVr1mDChAkAgLy8PJw4cQJZWVm2Lk7rw3xCREQuwOYBxd/fH126dNF7zNfXF6GhodrH77rrLsycORMhISEICAjAAw88gKysLPTp08fWxWmy9LjgxhdyQqxBISIiV2CXTrKNefXVV6FUKjFhwgRUVlYiJycHb731liOKYtLsEamOLoJFDDvFMp4QEZEraJGAsn79er3fvby8sHjxYixevLglNm+VcH8VzpVWwlfl5uiiNAlrUIiIyBXwXjwGRN3/CpnWRTCfEBGRK2BAMSCEFFGUMn1nGFCIiMgVyPQ0bD91+UQ2NSiGpWQTDxERuQIGFAPquoQi1/O8TItNRESkhwHFgKYPilxnZFXKteBEREQ6GFAMaJp45FIXYVjTI49SExERmceAYkDTxCPXigjeLJCIiFwBA4ohTSdZmZ7oZVpsIiIiPQwoBuReg8JRPERE5AoYUAzIbaI2w3LKNVgRERHpYkAxIP9hxjItOBERkQ4GFAPaidpkep6Xa7mJiIh0MaAY0DbxyPRML9NiExER6WFAMSBk1knWMJCwkywREbkCBhQDcrsXjyHmEyIicgUMKAY4zJiIiMjxGFAMyGym+wbkGqyIiIh0MaAYkFsTT8NSyqPcRERE5jCg6BD1dwqUbU2EXMtNRESkiwFFh04+kfEwY3mWm4iISBcDig41a1CIiIicAgOKDp0KFNn0QTEspmzKTUREZAYDig7dGhSFTN8ZtvAQEZErkOlp2D70+qA4rhjNomQbDxERuQAGFBPk2tlUnqUmIiLSx4CiQ46dZA37nHAmWSIicgUMKDr0m3jkeaJnPiEiIlfAgKJDr5OsTE/0ci03ERGRLgYUHXrDjGV6omcTDxERuQIGFB26TTxyOdEbFlMepSYiIjKPAUWH7r145Hqil0uwIiIiMocBRYdr3IvH0SUgIiJqPgYUHXIcZmxIrsGKiIhIFwOKDv1OsvI40RuWUibFJiIiMosBRYemBkXOJ3n2QSEiIlfAgKKrrgpFzqd4uTZNERER6WJA0aFp4pFTLYRhU5RcZ8AlIiLSxYCiwxWaeORcdiIiIg0GFB1C28Qj37M8AwoREbkCBhQdrlCDIqfmKSIiIlMYUHRoa1BkdI5vMMzYIaUgIiKyLQYUI+RcCyHnshMREWkwoOjQNvE4uBzNwXxCRESugAFFh6aJR861EHKZAZeIiMgcBhQdaiG/mdqYR4iIyBUxoOjQTNTGcz4REZFjMaDo0DbxyHS+eJkWm4iIqAEGFB1C5p1k2f+EiIhcBQOKDm0Tj4xO9Lqz3sqn1EREROYxoOiQYR9ZPXIefURERKSLAUWHgMynupdruYmIiAwwoBglzzO9PEtNRETUEAOKDk0Tj6zopBLZ1vwQEREZYEDRIcebBepiHxQiInIVDCg6tH1QHFyOppJruYmIiAwxoOiQYw2KnMpKRERkKZsHlEWLFqFXr17w9/dHeHg4xo4di7y8PL1lKioqMH36dISGhsLPzw8TJkxAYWGhrYvSZAoZ1UXoNuvIaf4WIiIic2weUDZs2IDp06dj8+bNWL16NaqrqzF8+HCUl5drl3n44Yfx3Xff4csvv8SGDRtw5swZjB8/3tZFsZosa1AcXQAiIiI7cLf1CletWqX3+7JlyxAeHo7t27dj4MCBKC4uxvvvv49PPvkEQ4cOBQAsXboUHTt2xObNm9GnTx9bF8lqcjrps2MsERG5Irv3QSkuLgYAhISEAAC2b9+O6upqZGdna5dJTU1FXFwcNm3aZHQdlZWVKCkp0ftnDwLyG2esm0+ELMdJExERNWTXgKJWqzFjxgz069cPXbp0AQAUFBTA09MTQUFBestGRESgoKDA6HoWLVqEwMBA7b/Y2Fi7lLe+iUc+tRKsQSEiIldk14Ayffp07N27F5999lmz1jN37lwUFxdr/508edJGJdQnx/oH5hMiInJFNu+DonH//ffj+++/x8aNGxETE6N9PDIyElVVVSgqKtKrRSksLERkZKTRdalUKqhUKnsVVUvTRCKnkz5rUIiIyBXZvAZFCIH7778fK1aswNq1a5GYmKj3fHp6Ojw8PLBmzRrtY3l5eThx4gSysrJsXRyraGpQ5HTOV8qorERERJayeQ3K9OnT8cknn+Cbb76Bv7+/tl9JYGAgvL29ERgYiLvuugszZ85ESEgIAgIC8MADDyArK8vhI3i0fVBkNI5Ht7+MHJuoiIiIjLF5QFmyZAkAYPDgwXqPL126FHfccQcA4NVXX4VSqcSECRNQWVmJnJwcvPXWW7YuShPIr4lHFwfxEBGRq7B5QLFkqKuXlxcWL16MxYsX23rzNiHTfAI3tvcQEZGL4L14dMi9BoL5hIiIXAUDio76TrLyPNOzBoWIiFwFA4qO+k6y8sQhx0RE5CoYUHQImScUJWtQiIjIRTCg6NA28Ti0FE3nxhoUIiJyEQwoOuR4Lx5dtXLv5UtERFSHAcUIecYT4HxppaOLQEREZBMMKDoE52IlIiJyCgwourRNPI4tBhERUWvHgKKjvpMsEwoREZEjMaDoEKxBISIicgoMKDrYB4WIiMg5MKDokPsw42AfD0cXgYiIyCYYUIyQZzwBRqZFOboIRERENsGAokPuDTycSZaIiFwFA4oOzb145Hqe5614iIjIVTCg6NAOM5bpiV6ufWeIiIgMMaDo0t7MWJ4neuYTIiJyFQwoOjTDjOV6olfKteBEREQGGFB0aIcZO7YYTebGTihEROQiGFCMkWlNhEyLTURE1AADig4h83HGbOIhIiJXwYCio/5mgfKUHhfs6CIQERHZhLujC+BM5DoPyvpHB+PA2RIM6xju6KIQERHZBAOKDrnWoCSE+SIhzNfRxSAiIrIZNvHokPvNAomIiFwFA4qeuiYeB5eCiIiotWNA0VFfg+LYchAREbV2DChGyHWqeyIiIlfBgKJD5tOgEBERuQwGFB1CrsN4iIiIXAwDig7BTrJEREROgQFFBzvJEhEROQcGFB31LTxMKERERI7EgKJDrlPdExERuRoGFCMYUIiIiByLAYWIiIicDgOKDm0nWfZBISIicigGFB3aYcbMJ0RERA7FgKJDcCpZIiIip8CAoqN+HhRWoRARETkSA4oOznRPRETkHBhQjGAFChERkWMxoOgQcuyEIgTwv7uBb+53dEmIiIhshgFFhyybeK5eBvZ8Aez4CCg77+jSEBER2QQDii45dpIV6vqfywocVw4iIiIbYkDR4Vt+DDe6rUeXq1sdXRTLqWvqfy4+5bhyEBER2RADio6owt/wsse7GFj6o6OLYjndgHLugOPKQUREZEMMKDpKfeMAAOE1ZxxcEivoBpQjax1XDiIiIhtiQNFR6lMXUKpPy2daWXVt/c/HfnNcOYiIiGyIAUVHmXdbXBWe8BIVQOFeRxfHMroBBQBKZFT7Q0REZAIDio5apTtOijbSL3mrHFsYS+k28QDsh0JERC6BAUVH//Zh8E7oLf0il+YSw4By5i/HlIOIiMiGGFB0xIf6InbcAumX/A1A0UnHFsgShgHl6AbHlIOIiMiGGFAMBccDYSnSz29lObYsltD2QambXO7Yb8CVSw4rDhERkS0woBiT/ZT0f1Up8NNjji1LYzQ1KKHtgJB20s//N0H6f9fnwPczG3akJSIicnIODSiLFy9GQkICvLy8kJmZiS1btjiyOPVSRwOxmdLPuW8D65533mHHmoCidAf6z5B+PvMX8HQgsGIqsO194PPbnLf8RERERjgsoHz++eeYOXMmnnrqKfz111/o1q0bcnJycO7cOUcVSd8UnVE8G14EFgRJJ/0l/YCSs85zwtcNKD3/YXyZvB+l8v/1EVBm8P7WVDZv+1cuAaWFzvN+uJofZ0nHXSnvs0RErYtCCMecWTIzM9GrVy+8+eabAAC1Wo3Y2Fg88MADmDNnjtnXlpSUIDAwEMXFxQgICLBfIStLgbf7A5ePWbZ8+2zAPxIoPg0c+x3IuBOIywSCE4CCPcDhNUBUNyAwBkjoD/iESsFCqKV/Sg9AoZCaZNw9pZO/uwrw8JEeN+bwr1KTTlQ34J6NQPkF4OV2zd/3NqnA5eNAzdX6x9pmAG1SAO9gIGUUsGyU/muueQboMh5w9wY8vKWOxodWA0FxUvku5wOhyUBML8DNEzi3X1pXTQXw3UPAyBeBgLaAdxCQ+450h+br35S2WXYO2PaBtJ4OIwBPH2mbQgDFJwG/COm9qq2W3tOrl6UyVBRL77PCTVpO5Q/4hNSXubYGKDkt9T2yhtDeWdL081VlgKefVBbvYNPLmlJTBTzbpv73eRcANw/r1mFPG14ConsACQOk915ON9lsCRXFwMUjQFgHQOVn323VVAF7vwIi04CILo77LH54BNj6X+liafhzgJcdv5/NOfCd9DfYfhjg6euYMjRGiIaf06ltwJZ3gf4zgdD2gJu7fcugVkvff+pqICTJvtuqY8352yEBpaqqCj4+Pvjqq68wduxY7eOTJ09GUVERvvnmG73lKysrUVlZf6VfUlKC2NhY+wcUjS3vAT8+av/tWCK0vRRmSguB6vL6xzUBRWPX51ITDwCpAy1rOJpMFQBUlthuff5RQOlZIDAWCEkECvcDVy5Iz8X1lQLNeSvnswmMAyI6AdVXpC+5iM5SUPNtI9WSHfq54WvcPIF2w6QgV3RCCs7VV+uXje4JiFop3FkyfN03XAoqxScB/2hp++pqKTReuQh0vA5QKOu+lBUm/of+Y6e2ABcP12+jTUfpvQmIkd5Dn1ApwHr6AmWFUsj3DpK+/MvPSc9fPCKFYJ9Q6UvYOxiAkC4ErlwEDv2ivx8dr5fCZdGJ+m13Hi9N3njhYP1ycVmAV5B0MVJVCngFSsdKsZnRfwkDgBObpROnV6D0WV06Uv+8hw+QfA3g7iXd/NMrUAq5NRXSeyLUgKe/dOGgcJNCUGUJcDK34bZCkqQLguKT0j77RQD7vm64XPJwwCdMWv/hX6X3MTlHChfqGqCyTPpc//5e/3WxfYDAtoCbSiqD4fMaEWnAhTzpM1FXS48pPaQLt4uHgLSbpOO/+JR0EeTmIb0v7p5AcKJ0MaJ3IVd30lbXSr/XVkmf1clcmPyuC0mSBj+4q6Tj3c1DqlGGAJKGSH8n1Vfqjg1IF6Xn9tcftz6h0nFcWy2991cuA37h0uOVpdI6NZ+PUNeFD6X0+quXpX2FQvocC3brH0cJA6TX5m9sWO42qdIx5RcOKN2AI+ukYz2yK+DXRjoGGoRRBVC4T/rbadMRCEuWXuPhIx0veT9K3z1BccDh1fovjekNXL0ERHWXjsHEAUC3icY/1yZy+oBy5swZtG3bFn/++SeysupHysyePRsbNmxAbq7+H9vTTz+NBQsWNFhPiwUUvY2ekiZDW7NAqhVxFl0mADd8YPr56qvA+kXAH/9puTIREZF8ZdwJXPuqTVdpTUCxc/2RbcydOxczZ87U/q6pQXGIwBjpX/I11r2utkaqrhNCSthXLkrT0ivdpKuUqnKp+aG0QLqyBaTk7eYBFOytv4oITgRqK4GKkrqquVopfGQ/bX77Ht7ANQulf5ZS1+pc8ersh7paugLwDZeatDTPV1dIV0IKJXC1CCg/L13Je9Q1x1RfAaCQHt+/EgjvBEBIVyWAtN9FJ6SfgxOkanFVgHSlLISU/pMGScGwoli6cgmKl64K26YD4R2l56rKpddo1hfdA/ANk15z+bhUvqA4YM8X0lXgvhXS1YJvqHSV5RUo1UYU7AGqrkj7WFUmlevCIal57uDP0jb3fAF0GgNsXwZ0u1X6PbyTdCXWfZJ09XrlkvQendomlSUoFji7W/o/JElqlsvfKL0XKaOkGo/NS6Rj7NAvwKSvgIBo4O8fpGNi39fSsRPavv4qLryj9Hmpa4Djf0plrKmQ3tvzfwN7v5Y+t+HPStXfXkHSMRHRWfr/6mXpaiwoTvpsK4qlZi+3uivOQ6ulK+qqcunYLT4NpN0gHa+XjkqjyAKipf5ZVy9L72VQPLBpsVRbcPAXoP9DUm2AENK+6v0PI48J6dje8KJ0JejhBXQYKd0UM3WUdLXfJlUqs9Jd+nfugNSkUlEk/Y2Un5euHiuKpWOg7JzUDKJQSv98QqXtrHpcer9iMqQraqWb9D4eWSM1V2hqIg6tlpoqu9wgHZ8BbYGNL0vbSBkNHP9dKuvBn+r/ZgLaSu9FQLT0mV44KP19BMZIx9PVy9K/cwek9fS6S/pbuXJJ2oZ/lLRv1Vel/fFtIx2vNRVS+Tz9pavl8vNAr38Cl/KlWpmek6X3u/SM9N0iBLBtKVBySipXZJr0N5I6WvpOUrgBuz6Vytfjdum4921T97dXJB1DF/Kk/fH0lZq0A2Ok47W0QNrOge+kJqa4LKl8SYOlfTy6XiqDVwCQ/xuQMlLap4uHpJo6T1+pZsEvAvCLlGqkik8B3iFSGRQK6fPS1Joo3eo/Q6Wb9PeQ+zaQeq1UltPbpH0PTpT2tcdt0uesUErvY9k5YMs70t9L73ukv6Pik/XvbfVVYOt7QLuh0vGkqV2pqZK+dypLpO817yDpd3eV1LSt+b7UNtWrpNeoq+uXqSgGNrwglavX3dL3gNIdOJ8nvf8DZ9X9LVdL5fAKBE5tlWq6Lh6Rjvuet0vvg1pt8KVd97d05RKQuwTo+4B0jFdX1O2/m/R95xUAxPeTvmc0zfjJw6Wa+KMbpL8vIaSaGgeSRROPoRbrg0JEREQ2Y8352yGjeDw9PZGeno41a9ZoH1Or1VizZo1ekw8RERG1Tg5r4pk5cyYmT56MjIwM9O7dG6+99hrKy8sxZcoURxWJiIiInITDAsrNN9+M8+fPY/78+SgoKED37t2xatUqREREOKpIRERE5CQcNg9Kc7APChERkfw4fR8UIiIiInMYUIiIiMjpMKAQERGR02FAISIiIqfDgEJEREROhwGFiIiInA4DChERETkdBhQiIiJyOgwoRERE5HQcNtV9c2gmvy0pKXFwSYiIiMhSmvO2JZPYyzKglJaWAgBiY2MdXBIiIiKyVmlpKQIDA80uI8t78ajVapw5cwb+/v5QKBQ2XXdJSQliY2Nx8uRJl7/PT2vaV6B17W9r2legde1va9pXoHXtb2vYVyEESktLER0dDaXSfC8TWdagKJVKxMTE2HUbAQEBLnuAGGpN+wq0rv1tTfsKtK79bU37CrSu/XX1fW2s5kSDnWSJiIjI6TCgEBERkdNhQDGgUqnw1FNPQaVSOboodtea9hVoXfvbmvYVaF3725r2FWhd+9ua9tUSsuwkS0RERK6NNShERETkdBhQiIiIyOkwoBAREZHTYUAhIiIip8OAomPx4sVISEiAl5cXMjMzsWXLFkcXyWpPP/00FAqF3r/U1FTt8xUVFZg+fTpCQ0Ph5+eHCRMmoLCwUG8dJ06cwOjRo+Hj44Pw8HDMmjULNTU1Lb0rRm3cuBHXXXcdoqOjoVAosHLlSr3nhRCYP38+oqKi4O3tjezsbBw6dEhvmUuXLmHSpEkICAhAUFAQ7rrrLpSVlekts3v3bgwYMABeXl6IjY3FSy+9ZO9da6Cxfb3jjjsafNYjRozQW0Yu+7po0SL06tUL/v7+CA8Px9ixY5GXl6e3jK2O3fXr16Nnz55QqVRo3749li1bZu/da8CS/R08eHCDz/fee+/VW0YO+7tkyRJ07dpVO/lYVlYWfvrpJ+3zrvS5Ao3vr6t8ri1CkBBCiM8++0x4enqKDz74QOzbt0/cfffdIigoSBQWFjq6aFZ56qmnROfOncXZs2e1/86fP699/t577xWxsbFizZo1Ytu2baJPnz6ib9++2udrampEly5dRHZ2ttixY4f48ccfRVhYmJg7d64jdqeBH3/8UTzxxBPi66+/FgDEihUr9J5/4YUXRGBgoFi5cqXYtWuXuP7660ViYqK4evWqdpkRI0aIbt26ic2bN4vffvtNtG/fXtxyyy3a54uLi0VERISYNGmS2Lt3r/j000+Ft7e3eOedd1pqN4UQje/r5MmTxYgRI/Q+60uXLuktI5d9zcnJEUuXLhV79+4VO3fuFKNGjRJxcXGirKxMu4wtjt2jR48KHx8fMXPmTLF//37xxhtvCDc3N7Fq1Sqn299BgwaJu+++W+/zLS4ult3+fvvtt+KHH34QBw8eFHl5eeLxxx8XHh4eYu/evUII1/pcLdlfV/lcWwIDSp3evXuL6dOna3+vra0V0dHRYtGiRQ4slfWeeuop0a1bN6PPFRUVCQ8PD/Hll19qHztw4IAAIDZt2iSEkE6KSqVSFBQUaJdZsmSJCAgIEJWVlXYtu7UMT9pqtVpERkaKl19+WftYUVGRUKlU4tNPPxVCCLF//34BQGzdulW7zE8//SQUCoU4ffq0EEKIt956SwQHB+vt72OPPSZSUlLsvEemmQooY8aMMfkaue6rEEKcO3dOABAbNmwQQtju2J09e7bo3Lmz3rZuvvlmkZOTY+9dMstwf4WQTmQPPfSQydfIeX+Dg4PFf//7X5f/XDU0+yuEa3+utsYmHgBVVVXYvn07srOztY8plUpkZ2dj06ZNDixZ0xw6dAjR0dFISkrCpEmTcOLECQDA9u3bUV1drbefqampiIuL0+7npk2bkJaWhoiICO0yOTk5KCkpwb59+1p2R6yUn5+PgoICvf0LDAxEZmam3v4FBQUhIyNDu0x2djaUSiVyc3O1ywwcOBCenp7aZXJycpCXl4fLly+30N5YZv369QgPD0dKSgqmTZuGixcvap+T874WFxcDAEJCQgDY7tjdtGmT3jo0yzj679xwfzU+/vhjhIWFoUuXLpg7dy6uXLmifU6O+1tbW4vPPvsM5eXlyMrKcvnP1XB/NVztc7UXWd4s0NYuXLiA2tpavQMCACIiIvD33387qFRNk5mZiWXLliElJQVnz57FggULMGDAAOzduxcFBQXw9PREUFCQ3msiIiJQUFAAACgoKDD6Pmiec2aa8hkrv+7+hYeH6z3v7u6OkJAQvWUSExMbrEPzXHBwsF3Kb60RI0Zg/PjxSExMxJEjR/D4449j5MiR2LRpE9zc3GS7r2q1GjNmzEC/fv3QpUsXbVlsceyaWqakpARXr16Ft7e3PXbJLGP7CwC33nor4uPjER0djd27d+Oxxx5DXl4evv76awDy2t89e/YgKysLFRUV8PPzw4oVK9CpUyfs3LnTJT9XU/sLuNbnam8MKC5m5MiR2p+7du2KzMxMxMfH44svvnCZg5YkEydO1P6clpaGrl27ol27dli/fj2GDRvmwJI1z/Tp07F37178/vvvji5KizC1v1OnTtX+nJaWhqioKAwbNgxHjhxBu3btWrqYzZKSkoKdO3eiuLgYX331FSZPnowNGzY4ulh2Y2p/O3Xq5FKfq72xiQdAWFgY3NzcGvQcLywsRGRkpINKZRtBQUHo0KEDDh8+jMjISFRVVaGoqEhvGd39jIyMNPo+aJ5zZprymfscIyMjce7cOb3na2pqcOnSJdm/B0lJSQgLC8Phw4cByHNf77//fnz//fdYt24dYmJitI/b6tg1tUxAQIBDAryp/TUmMzMTAPQ+X7nsr6enJ9q3b4/09HQsWrQI3bp1w3/+8x+X/VxN7a8xcv5c7Y0BBdLBlJ6ejjVr1mgfU6vVWLNmjV67oRyVlZXhyJEjiIqKQnp6Ojw8PPT2My8vDydOnNDuZ1ZWFvbs2aN3Ylu9ejUCAgK0VZTOKjExEZGRkXr7V1JSgtzcXL39Kyoqwvbt27XLrF27Fmq1WvtFkZWVhY0bN6K6ulq7zOrVq5GSkuI0zTvGnDp1ChcvXkRUVBQAee2rEAL3338/VqxYgbVr1zZodrLVsZuVlaW3Ds0yLf133tj+GrNz504A0Pt85bK/htRqNSorK13uczVFs7/GuNLnanOO7qXrLD777DOhUqnEsmXLxP79+8XUqVNFUFCQXk9qOXjkkUfE+vXrRX5+vvjjjz9Edna2CAsLE+fOnRNCSEP64uLixNq1a8W2bdtEVlaWyMrK0r5eM8Rt+PDhYufOnWLVqlWiTZs2TjPMuLS0VOzYsUPs2LFDABCvvPKK2LFjhzh+/LgQQhpmHBQUJL755huxe/duMWbMGKPDjHv06CFyc3PF77//LpKTk/WG3hYVFYmIiAhx++23i71794rPPvtM+Pj4tPjQW3P7WlpaKh599FGxadMmkZ+fL3799VfRs2dPkZycLCoqKmS3r9OmTROBgYFi/fr1esMvr1y5ol3GFseuZnjmrFmzxIEDB8TixYsdMjyzsf09fPiwWLhwodi2bZvIz88X33zzjUhKShIDBw6U3f7OmTNHbNiwQeTn54vdu3eLOXPmCIVCIX755RchhGt9ro3tryt9ri2BAUXHG2+8IeLi4oSnp6fo3bu32Lx5s6OLZLWbb75ZREVFCU9PT9G2bVtx8803i8OHD2ufv3r1qrjvvvtEcHCw8PHxEePGjRNnz57VW8exY8fEyJEjhbe3twgLCxOPPPKIqK6ubuldMWrdunUCQIN/kydPFkJIQ43nzZsnIiIihEqlEsOGDRN5eXl667h48aK45ZZbhJ+fnwgICBBTpkwRpaWlesvs2rVL9O/fX6hUKtG2bVvxwgsvtNQuapnb1ytXrojhw4eLNm3aCA8PDxEfHy/uvvvuBoFaLvtqbD8BiKVLl2qXsdWxu27dOtG9e3fh6ekpkpKS9LbRUhrb3xMnToiBAweKkJAQoVKpRPv27cWsWbP05ssQQh77e+edd4r4+Hjh6ekp2rRpI4YNG6YNJ0K41ucqhPn9daXPtSUohBCi5epriIiIiBrHPihERETkdBhQiIiIyOkwoBAREZHTYUAhIiIip8OAQkRERE6HAYWIiIicDgMKEREROR0GFCIiInI6DChERETkdBhQiIiIyOkwoBAREZHTYUAhIiIip/P/OSQdzXYl9CAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "train_loss = np.array(train_loss)\n",
    "train_accs = np.array(train_accs)\n",
    "plt.plot(train_accs)\n",
    "plt.plot(train_loss)\n",
    "np.savetxt('train_loss2.csv', train_loss, delimiter=\",\")\n",
    "np.savetxt('train_accs2.csv', train_accs, delimiter=\",\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 (default, Oct 19 2022, 22:38:03) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "91eaa33755ee0e6c8927d7837736bb7bf44cb27baec87b08c7a4a0a98fc82110"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
